# base encoder settings that can be extended and overriden
# we leave out the attention part for other config to override

_target_: xformers.factory.block_factory.xFormerEncoderConfig
reversible: False
num_layers: 4
use_triton: True
dim_model: ${emb}
residual_norm_style: pre
position_encoding_config:
  name: vocab
  seq_len: 1024
  vocab_size: ${vocab}
  dropout: 0
multi_head_config:
  num_heads: 4
  residual_dropout: 0
  attention: ???
feedforward_config:
  name: MLP
  dropout: 0
  activation: relu
  hidden_layer_multiplier: 4
